{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "microdistilmodel.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/gagan3012/project-code-py/blob/master/notebooks/gptneomodel.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SWlrvhB-hO_c",
        "outputId": "1f5ed63c-dcbd-4806-a538-f6d332b0fec4"
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tue May 25 03:15:23 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   37C    P8     9W /  70W |      0MiB / 15109MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lsTTbIxoSXTc",
        "outputId": "8c9eeed2-e4a6-431b-c153-b4826662ad63"
      },
      "source": [
        "lines = ['https://github.com/Garvit244/Leetcode',\n",
        "         'https://github.com/shichao-an/leetcode-python',\n",
        "         'https://github.com/algorhythms/LeetCode',\n",
        "         'https://github.com/wuduhren/leetcode-python',\n",
        "         'https://github.com/csujedihy/lc-all-solutions',\n",
        "         'https://github.com/vJechsmayr/PythonAlgorithms',\n",
        "         'https://github.com/HuberTRoy/leetCode',\n",
        "         'https://github.com/qiyuangong/leetcode',\n",
        "         'https://github.com/MTrajK/coding-problems',\n",
        "         'https://github.com/JushuangQiao/Python-LeetCode',\n",
        "         'https://github.com/Jack-Lee-Hiter/AlgorithmsByPython',\n",
        "         'https://github.com/sapanz/Hackerrank-Problem-Solving-Python-Solutions',\n",
        "         'https://github.com/arsho/Hackerrank_Python_Domain_Solutions',\n",
        "         'https://github.com/swapnanildutta/Hackerrank-Codes',\n",
        "         'https://github.com/markopuza/Competitive-programming-in-Python',\n",
        "         'https://github.com/deepaksood619/Python-Competitive-Programming',\n",
        "         'https://github.com/ndb796/Python-Competitive-Programming-Team-Notes',\n",
        "         'https://github.com/harshitbansal373/python',\n",
        "         'https://github.com/yashagrawal300/python-programs',\n",
        "         'https://github.com/bmegha98/Python-Practice',\n",
        "         'https://github.com/geekcomputers/Python',\n",
        "         'https://github.com/smilejay/python',\n",
        "         'https://github.com/yuzhoujr/leetcode',\n",
        "         'https://github.com/franklingu/leetcode-solutions',\n",
        "         'https://github.com/kumailn/Algorithms',\n",
        "         'https://github.com/Diego-Zulu/leetcode_answers',\n",
        "         'https://github.com/concealedtea/Coding-Interview-Prep',\n",
        "         'https://github.com/Wang-Yann/LeetCodeMe',\n",
        "         'https://github.com/hwm18/MyLeetCode',\n",
        "         'https://github.com/lixiang2017/leetcode',\n",
        "         'https://github.com/thisisshub/DSA',\n",
        "         'https://github.com/criszhou/LeetCode-Python',\n",
        "         'https://github.com/lilianweng/LeetcodePython',\n",
        "         'https://github.com/jioyoung/leetcode',\n",
        "         'https://github.com/Vikktour/Data-Structures-Algorithms-Implementations',\n",
        "         'https://github.com/lkwq007/leetcode-py',\n",
        "         'https://github.com/yz5308/Python_Leetcode',\n",
        "         'https://github.com/Garvit244/Leetcode',\n",
        "         'https://github.com/duanzhihao2017/Leetcode']\n",
        "\n",
        "len(lines)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "39"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8ArghELG6QZ5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "368e23aa-d4cb-4cac-b2cf-0a7abc39e465"
      },
      "source": [
        "from subprocess import call\n",
        "import math\n",
        "import os\n",
        "import csv\n",
        "csv_columns = ['text']\n",
        "\n",
        "\n",
        "for line in lines:\n",
        "    call(['git', 'clone', line.strip(), f'resources/{line.strip().split(\"/\")[-1]}'])\n",
        "\n",
        "json_data = []\n",
        "total_files = []\n",
        "count = 0\n",
        "\n",
        "for line in lines:\n",
        "    for currentpath, folders, files in os.walk(f'resources/{line.strip().split(\"/\")[-1]}'):\n",
        "        for file in files:\n",
        "            if file[-3:] == '.py':\n",
        "                count += 1\n",
        "                total_files.append(os.path.join(currentpath, file))\n",
        "\n",
        "print('files: ', len(total_files))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "files:  11352\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JR-q5mlSAWxJ"
      },
      "source": [
        "for file in total_files:\n",
        "    with open(file, \"r\") as f:\n",
        "        try:\n",
        "            t = f.readlines()\n",
        "        except UnicodeDecodeError:\n",
        "            print('DecoderError: ', file)\n",
        "        summary = ''.join(t)\n",
        "        summary = str(summary).strip()\n",
        "        bos_token = '<|title|>'\n",
        "        eos_token = '<|endoftext|>'\n",
        "        data = bos_token + summary + eos_token\n",
        "        json_data.append({'text': data})"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JHdBhEpR6cfD"
      },
      "source": [
        "with open(\"data.csv\", 'w') as csvfile:\n",
        "        writer = csv.DictWriter(csvfile, fieldnames=csv_columns)\n",
        "        writer.writeheader()\n",
        "        for data in json_data:\n",
        "            writer.writerow(data)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tH8MJYkm8Ldt"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "df = pd.read_csv('/content/data.csv')\n",
        "\n",
        "!rm -rf resources"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oeTMr6_W-o4x",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "outputId": "c77941ca-51fd-4049-96ea-139d35ecc4b4"
      },
      "source": [
        "df['text'][1]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'<|title|>\\'\\'\\'\\nIn a string S of lowercase letters, these letters form consecutive groups of the same character.\\n\\nFor example, a string like S = \"abbxxxxzyy\" has the groups \"a\", \"bb\", \"xxxx\", \"z\" and \"yy\".\\n\\nCall a group large if it has 3 or more characters.  We would like the starting and ending positions of every large group.\\n\\nThe final answer should be in lexicographic order.\\n\\n \\n\\nExample 1:\\n\\nInput: \"abbxxxxzzy\"\\nOutput: [[3,6]]\\nExplanation: \"xxxx\" is the single large group with starting  3 and ending positions 6.\\nExample 2:\\n\\nInput: \"abc\"\\nOutput: []\\nExplanation: We have \"a\",\"b\" and \"c\" but no large group.\\nExample 3:\\n\\nInput: \"abcdddeeeeaabbbcd\"\\nOutput: [[3,5],[6,9],[12,14]]\\n\\'\\'\\'\\n\\nclass Solution(object):\\n    def largeGroupPositions(self, S):\\n        \"\"\"\\n        :type S: str\\n        :rtype: List[List[int]]\\n        \"\"\"\\n        if not S:\\n            return []\\n        \\n        result = []\\n        count = 1\\n        prevChar = S[0]\\n        index_i = 0\\n        for index in range(1,len(S)):\\n            if S[index] == prevChar:\\n                count += 1\\n            else:\\n                if count >= 3:\\n                    result.append([index_i, index-1])\\n                \\n                count = 1\\n                prevChar = S[index]\\n                index_i = index\\n                \\n        if count >= 3:\\n            result.append([index_i, len(S)-1])\\n        return result<|endoftext|>'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EAmmNuZOUqeY",
        "outputId": "88601bd6-14d9-4134-9335-f6c89c7914e3"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "train, eval = train_test_split(df, train_size=.8, random_state=2020)\n",
        "print(len(train))\n",
        "print(len(eval))\n",
        "\n",
        "train = train['text'].tolist()\n",
        "eval = eval['text'].tolist()\n",
        "\n",
        "\n",
        "with open('train_tmp.txt', 'w') as file_handle:\n",
        "  file_handle.write(str(train))\n",
        "\n",
        "with open('eval_tmp.txt', 'w') as file_handle:\n",
        "  file_handle.write(str(eval))"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "9081\n",
            "2271\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRhnHXcFZQVc"
      },
      "source": [
        "!rm data.csv"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8JxD4TxQ-qCt",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "196e1f45-77f5-4426-9330-53ac58f0fbac"
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers\n",
        "!pip install transformers -U\n",
        "!pip install datasets\n",
        "!pip install wandb"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'transformers'...\n",
            "remote: Enumerating objects: 73337, done.\u001b[K\n",
            "remote: Counting objects: 100% (93/93), done.\u001b[K\n",
            "remote: Compressing objects: 100% (62/62), done.\u001b[K\n",
            "remote: Total 73337 (delta 37), reused 63 (delta 23), pack-reused 73244\u001b[K\n",
            "Receiving objects: 100% (73337/73337), 56.31 MiB | 26.21 MiB/s, done.\n",
            "Resolving deltas: 100% (52133/52133), done.\n",
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d5/43/cfe4ee779bbd6a678ac6a97c5a5cdeb03c35f9eaebbb9720b036680f9a2d/transformers-4.6.1-py3-none-any.whl (2.2MB)\n",
            "\u001b[K     |████████████████████████████████| 2.3MB 8.7MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.0.1)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Collecting huggingface-hub==0.0.8\n",
            "  Downloading https://files.pythonhosted.org/packages/a1/88/7b1e45720ecf59c6c6737ff332f41c955963090a18e72acbcbeac6b25e86/huggingface_hub-0.0.8-py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 34.5MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/e2/df3543e8ffdab68f5acc73f613de9c2b155ac47f162e725dcac87c521c11/tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 34.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (8.0.0)\n",
            "Installing collected packages: huggingface-hub, sacremoses, tokenizers, transformers\n",
            "Successfully installed huggingface-hub-0.0.8 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.6.1\n",
            "Collecting datasets\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/46/1a/b9f9b3bfef624686ae81c070f0a6bb635047b17cdb3698c7ad01281e6f9a/datasets-1.6.2-py3-none-any.whl (221kB)\n",
            "\u001b[K     |████████████████████████████████| 225kB 6.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets) (4.0.1)\n",
            "Collecting fsspec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/bc/52/816d1a3a599176057bf29dfacb1f8fadb61d35fbd96cb1bab4aaa7df83c0/fsspec-2021.5.0-py3-none-any.whl (111kB)\n",
            "\u001b[K     |████████████████████████████████| 112kB 18.2MB/s \n",
            "\u001b[?25hCollecting xxhash\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/4f/0a862cad26aa2ed7a7cd87178cbbfa824fc1383e472d63596a0d018374e7/xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243kB)\n",
            "\u001b[K     |████████████████████████████████| 245kB 17.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.3)\n",
            "Requirement already satisfied: huggingface-hub<0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.0.8)\n",
            "Requirement already satisfied: pyarrow>=1.0.0<4.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (20.9)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.41.1)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2020.12.5)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (2.4.7)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: fsspec, xxhash, datasets\n",
            "Successfully installed datasets-1.6.2 fsspec-2021.5.0 xxhash-2.0.2\n",
            "Collecting wandb\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/5f/45439b4767334b868e1c8c35b1b0ba3747d8c21be77b79f09eed7aa3c72b/wandb-0.10.30-py2.py3-none-any.whl (1.8MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8MB 8.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (8.0.0)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Collecting GitPython>=1.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/da/6f6224fdfc47dab57881fe20c0d1bc3122be290198ba0bf26a953a045d92/GitPython-3.1.17-py3-none-any.whl (166kB)\n",
            "\u001b[K     |████████████████████████████████| 174kB 50.5MB/s \n",
            "\u001b[?25hCollecting sentry-sdk>=0.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1c/4a/a54b254f67d8f4052338d54ebe90126f200693440a93ef76d254d581e3ec/sentry_sdk-1.1.0-py2.py3-none-any.whl (131kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 52.1MB/s \n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f5/e8/f6bd1eee09314e7e6dee49cbe2c5e22314ccdb38db16c9fc72d2fa80d054/docker_pycreds-0.4.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Collecting subprocess32>=3.5.3\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/32/c8/564be4d12629b912ea431f1a50eb8b3b9d00f1a0b1ceff17f266be190007/subprocess32-3.5.4.tar.gz (97kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 13.5MB/s \n",
            "\u001b[?25hCollecting configparser>=3.8.1\n",
            "  Downloading https://files.pythonhosted.org/packages/fd/01/ff260a18caaf4457eb028c96eeb405c4a230ca06c8ec9c1379f813caa52e/configparser-5.0.2-py3-none-any.whl\n",
            "Collecting shortuuid>=0.5.0\n",
            "  Downloading https://files.pythonhosted.org/packages/25/a6/2ecc1daa6a304e7f1b216f0896b26156b78e7c38e1211e9b798b4716c53d/shortuuid-1.0.1-py3-none-any.whl\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.12.4)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (3.13)\n",
            "Collecting pathtools\n",
            "  Downloading https://files.pythonhosted.org/packages/e7/7f/470d6fcdf23f9f3518f6b0b76be9df16dcc8630ad409947f8be2eb0ed13a/pathtools-0.1.2.tar.gz\n",
            "Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)\n",
            "Collecting gitdb<5,>=4.0.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ea/e8/f414d1a4f0bbc668ed441f74f44c116d9816833a48bf81d22b697090dba8/gitdb-4.0.7-py3-none-any.whl (63kB)\n",
            "\u001b[K     |████████████████████████████████| 71kB 9.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.0; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb) (3.7.4.3)\n",
            "Requirement already satisfied: urllib3>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from sentry-sdk>=0.4.0->wandb) (1.24.3)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from sentry-sdk>=0.4.0->wandb) (2020.12.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.12.0->wandb) (56.1.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Collecting smmap<5,>=3.0.1\n",
            "  Downloading https://files.pythonhosted.org/packages/68/ee/d540eb5e5996eb81c26ceffac6ee49041d473bc5125f2aa995cf51ec1cf1/smmap-4.0.0-py2.py3-none-any.whl\n",
            "Building wheels for collected packages: subprocess32, pathtools\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for subprocess32: filename=subprocess32-3.5.4-cp37-none-any.whl size=6489 sha256=ba69ff22c9f3c42201a184885f425045d098781195c715812eca9575550a6cbb\n",
            "  Stored in directory: /root/.cache/pip/wheels/68/39/1a/5e402bdfdf004af1786c8b853fd92f8c4a04f22aad179654d1\n",
            "  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pathtools: filename=pathtools-0.1.2-cp37-none-any.whl size=8786 sha256=435c8d55493e8fc5fa7870be3b6bb2ccd813cf4738db4b90f36e9154c49277f8\n",
            "  Stored in directory: /root/.cache/pip/wheels/0b/04/79/c3b0c3a0266a3cb4376da31e5bfe8bba0c489246968a68e843\n",
            "Successfully built subprocess32 pathtools\n",
            "Installing collected packages: smmap, gitdb, GitPython, sentry-sdk, docker-pycreds, subprocess32, configparser, shortuuid, pathtools, wandb\n",
            "Successfully installed GitPython-3.1.17 configparser-5.0.2 docker-pycreds-0.4.0 gitdb-4.0.7 pathtools-0.1.2 sentry-sdk-1.1.0 shortuuid-1.0.1 smmap-4.0.0 subprocess32-3.5.4 wandb-0.10.30\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ojIGLxZU1eK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa3d4fea-a545-4bbf-9d5f-15c93d73f1ed"
      },
      "source": [
        "import os\n",
        "os.chdir(\"/content/transformers/\")\n",
        "!pip install .\n",
        "!pwd"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Processing /content/transformers\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: huggingface-hub==0.0.8 in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (0.0.8)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (4.0.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (20.9)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (0.0.45)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (3.0.12)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (1.19.5)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers==4.7.0.dev0) (0.10.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.7.0.dev0) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.7.0.dev0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.7.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.7.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.7.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers==4.7.0.dev0) (3.4.1)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.7.0.dev0) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.7.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.7.0.dev0) (1.0.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.7.0.dev0) (8.0.0)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.7.0.dev0-cp37-none-any.whl size=2309125 sha256=4f769d2d6e150a4c9d194c3676d52fd214844b98da0d581491586ff5d18d8704\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-yxx594t_/wheels/23/19/dd/2561a4e47240cf6b307729d58e56f8077dd0c698f5992216cf\n",
            "Successfully built transformers\n",
            "Installing collected packages: transformers\n",
            "  Found existing installation: transformers 4.6.1\n",
            "    Uninstalling transformers-4.6.1:\n",
            "      Successfully uninstalled transformers-4.6.1\n",
            "Successfully installed transformers-4.7.0.dev0\n",
            "/content/transformers\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yhF9yjagBBNh",
        "outputId": "2d8ea4fd-5b01-4fec-bfa2-821829e2396d"
      },
      "source": [
        "os.chdir(\"/content/transformers/examples/pytorch/\")\n",
        "os.chdir(\"./language-modeling\")\n",
        "!pwd\n",
        "!pip install -r requirements.txt"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/transformers/examples/pytorch/language-modeling\n",
            "Requirement already satisfied: datasets>=1.1.3 in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 1)) (1.6.2)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 8.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (from -r requirements.txt (line 3)) (3.12.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (1.19.5)\n",
            "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (4.41.1)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (2.23.0)\n",
            "Requirement already satisfied: huggingface-hub<0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.0.8)\n",
            "Requirement already satisfied: pyarrow>=1.0.0<4.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (3.0.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.3.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (1.1.5)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (2021.5.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (20.9)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (2.0.2)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (4.0.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets>=1.1.3->-r requirements.txt (line 1)) (0.70.11.1)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf->-r requirements.txt (line 3)) (1.15.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf->-r requirements.txt (line 3)) (56.1.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets>=1.1.3->-r requirements.txt (line 1)) (1.24.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets>=1.1.3->-r requirements.txt (line 1)) (3.0.12)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets>=1.1.3->-r requirements.txt (line 1)) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets>=1.1.3->-r requirements.txt (line 1)) (2018.9)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets>=1.1.3->-r requirements.txt (line 1)) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets>=1.1.3->-r requirements.txt (line 1)) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->datasets>=1.1.3->-r requirements.txt (line 1)) (3.7.4.3)\n",
            "Installing collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "9Z51w4XLXjh-",
        "outputId": "6a086046-5bb5-4290-a11a-3dcdaf4f810b"
      },
      "source": [
        "import wandb\n",
        "wandb.login()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vh75u-DGFXaO",
        "outputId": "c0112273-2417-40c5-8831-62f74047b602"
      },
      "source": [
        "%env WANDB_PROJECT=project-code-py\n",
        "\n",
        "!python run_clm.py \\\n",
        "--model_type EleutherAI/gpt-neo-125M \\\n",
        "--model_name_or_path EleutherAI/gpt-neo-125M \\\n",
        "--train_file \"/content/train_tmp.txt\" \\\n",
        "--do_train \\\n",
        "--validation_file \"/content/eval_tmp.txt\" \\\n",
        "--do_eval \\\n",
        "--per_device_train_batch_size 1 \\\n",
        "--per_device_eval_batch_size 1 \\\n",
        "--save_steps -1 \\\n",
        "--num_train_epochs 5 \\\n",
        "--fp16 \\\n",
        "--output_dir=\"/content/model\" \\\n",
        "--report_to wandb "
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "env: WANDB_PROJECT=project-code-py\n",
            "2021-05-25 03:17:02.102850: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "05/25/2021 03:17:03 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: True\n",
            "05/25/2021 03:17:03 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir=/content/model, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.NO, prediction_loss_only=False, per_device_train_batch_size=1, per_device_eval_batch_size=1, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=5.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=0, logging_dir=runs/May25_03-17-03_76648a7ce70b, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=-1, save_total_limit=None, no_cuda=False, seed=42, fp16=True, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=-1, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=500, dataloader_num_workers=0, past_index=-1, run_name=/content/model, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=None, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['wandb'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=True, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, _n_gpu=1, mp_parameters=)\n",
            "05/25/2021 03:17:04 - WARNING - datasets.builder -   Using custom data configuration default-7aa8480df51aaa75\n",
            "Downloading and preparing dataset text/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /root/.cache/huggingface/datasets/text/default-7aa8480df51aaa75/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5...\n",
            "Dataset text downloaded and prepared to /root/.cache/huggingface/datasets/text/default-7aa8480df51aaa75/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5. Subsequent calls will reuse this data.\n",
            "[INFO|file_utils.py:1556] 2021-05-25 03:17:04,784 >> https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpcoy15imf\n",
            "Downloading: 100% 1.01k/1.01k [00:00<00:00, 1.09MB/s]\n",
            "[INFO|file_utils.py:1560] 2021-05-25 03:17:04,992 >> storing https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/config.json in cache at /root/.cache/huggingface/transformers/29380fef22a43cbfb3d3a6c8e2f4fd951459584d87c34e4621b30580a54aca84.350b2531c3ac33139f960e42aa0e8f72fed7248de2cc97c4e34ad41c00234396\n",
            "[INFO|file_utils.py:1568] 2021-05-25 03:17:04,992 >> creating metadata file for /root/.cache/huggingface/transformers/29380fef22a43cbfb3d3a6c8e2f4fd951459584d87c34e4621b30580a54aca84.350b2531c3ac33139f960e42aa0e8f72fed7248de2cc97c4e34ad41c00234396\n",
            "[INFO|configuration_utils.py:517] 2021-05-25 03:17:04,992 >> loading configuration file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/29380fef22a43cbfb3d3a6c8e2f4fd951459584d87c34e4621b30580a54aca84.350b2531c3ac33139f960e42aa0e8f72fed7248de2cc97c4e34ad41c00234396\n",
            "[INFO|configuration_utils.py:553] 2021-05-25 03:17:04,993 >> Model config GPTNeoConfig {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPTNeoForCausalLM\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0,\n",
            "  \"attention_layers\": [\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\"\n",
            "  ],\n",
            "  \"attention_types\": [\n",
            "    [\n",
            "      [\n",
            "        \"global\",\n",
            "        \"local\"\n",
            "      ],\n",
            "      6\n",
            "    ]\n",
            "  ],\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embed_dropout\": 0,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": null,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"max_position_embeddings\": 2048,\n",
            "  \"model_type\": \"gpt_neo\",\n",
            "  \"num_heads\": 12,\n",
            "  \"num_layers\": 12,\n",
            "  \"resid_dropout\": 0,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"transformers_version\": \"4.7.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257,\n",
            "  \"window_size\": 256\n",
            "}\n",
            "\n",
            "[INFO|configuration_utils.py:517] 2021-05-25 03:17:05,196 >> loading configuration file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/29380fef22a43cbfb3d3a6c8e2f4fd951459584d87c34e4621b30580a54aca84.350b2531c3ac33139f960e42aa0e8f72fed7248de2cc97c4e34ad41c00234396\n",
            "[INFO|configuration_utils.py:553] 2021-05-25 03:17:05,197 >> Model config GPTNeoConfig {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPTNeoForCausalLM\"\n",
            "  ],\n",
            "  \"attention_dropout\": 0,\n",
            "  \"attention_layers\": [\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\",\n",
            "    \"global\",\n",
            "    \"local\"\n",
            "  ],\n",
            "  \"attention_types\": [\n",
            "    [\n",
            "      [\n",
            "        \"global\",\n",
            "        \"local\"\n",
            "      ],\n",
            "      6\n",
            "    ]\n",
            "  ],\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embed_dropout\": 0,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": null,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"max_position_embeddings\": 2048,\n",
            "  \"model_type\": \"gpt_neo\",\n",
            "  \"num_heads\": 12,\n",
            "  \"num_layers\": 12,\n",
            "  \"resid_dropout\": 0,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"transformers_version\": \"4.7.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257,\n",
            "  \"window_size\": 256\n",
            "}\n",
            "\n",
            "[INFO|file_utils.py:1556] 2021-05-25 03:17:05,410 >> https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/vocab.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp5bm3s7zu\n",
            "Downloading: 100% 899k/899k [00:00<00:00, 2.40MB/s]\n",
            "[INFO|file_utils.py:1560] 2021-05-25 03:17:05,996 >> storing https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/vocab.json in cache at /root/.cache/huggingface/transformers/08c00c4159e921d4c941ac75732643373aba509d9b352a82bbbb043a94058d98.a552555fdda56a1c7c9a285bccfd44ac8e4b9e26c8c9b307831b3ea3ac782b45\n",
            "[INFO|file_utils.py:1568] 2021-05-25 03:17:05,996 >> creating metadata file for /root/.cache/huggingface/transformers/08c00c4159e921d4c941ac75732643373aba509d9b352a82bbbb043a94058d98.a552555fdda56a1c7c9a285bccfd44ac8e4b9e26c8c9b307831b3ea3ac782b45\n",
            "[INFO|file_utils.py:1556] 2021-05-25 03:17:06,201 >> https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/merges.txt not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp89yohsjq\n",
            "Downloading: 100% 456k/456k [00:00<00:00, 1.46MB/s]\n",
            "[INFO|file_utils.py:1560] 2021-05-25 03:17:06,720 >> storing https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/merges.txt in cache at /root/.cache/huggingface/transformers/12305762709d884a770efe7b0c68a7f4bc918da44e956058d43da0d12f7bea20.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1568] 2021-05-25 03:17:06,720 >> creating metadata file for /root/.cache/huggingface/transformers/12305762709d884a770efe7b0c68a7f4bc918da44e956058d43da0d12f7bea20.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|file_utils.py:1556] 2021-05-25 03:17:07,487 >> https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/special_tokens_map.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpwm9da04d\n",
            "Downloading: 100% 357/357 [00:00<00:00, 440kB/s]\n",
            "[INFO|file_utils.py:1560] 2021-05-25 03:17:07,700 >> storing https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/special_tokens_map.json in cache at /root/.cache/huggingface/transformers/6c3239a63aaf46ec7625b38abfe41fc2ce0b25f90800aefe6526256340d4ab6d.2b8bf81243d08385c806171bc7ced6d2a0dcc7f896ca637f4e777418f7f0cc3c\n",
            "[INFO|file_utils.py:1568] 2021-05-25 03:17:07,700 >> creating metadata file for /root/.cache/huggingface/transformers/6c3239a63aaf46ec7625b38abfe41fc2ce0b25f90800aefe6526256340d4ab6d.2b8bf81243d08385c806171bc7ced6d2a0dcc7f896ca637f4e777418f7f0cc3c\n",
            "[INFO|file_utils.py:1556] 2021-05-25 03:17:07,912 >> https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/tokenizer_config.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmp6jky40ui\n",
            "Downloading: 100% 560/560 [00:00<00:00, 652kB/s]\n",
            "[INFO|file_utils.py:1560] 2021-05-25 03:17:08,125 >> storing https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/tokenizer_config.json in cache at /root/.cache/huggingface/transformers/3cc88b3aa29bb2546db2dc21783292e2a086bb7158c7b5ceddeb24158a85c183.e74f7c3643ee79eb023ead36008be72fe726dada60fa3b2a0569925cfefa1e74\n",
            "[INFO|file_utils.py:1568] 2021-05-25 03:17:08,125 >> creating metadata file for /root/.cache/huggingface/transformers/3cc88b3aa29bb2546db2dc21783292e2a086bb7158c7b5ceddeb24158a85c183.e74f7c3643ee79eb023ead36008be72fe726dada60fa3b2a0569925cfefa1e74\n",
            "[INFO|tokenization_utils_base.py:1717] 2021-05-25 03:17:08,125 >> loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/08c00c4159e921d4c941ac75732643373aba509d9b352a82bbbb043a94058d98.a552555fdda56a1c7c9a285bccfd44ac8e4b9e26c8c9b307831b3ea3ac782b45\n",
            "[INFO|tokenization_utils_base.py:1717] 2021-05-25 03:17:08,125 >> loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/12305762709d884a770efe7b0c68a7f4bc918da44e956058d43da0d12f7bea20.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1717] 2021-05-25 03:17:08,125 >> loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/tokenizer.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1717] 2021-05-25 03:17:08,125 >> loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1717] 2021-05-25 03:17:08,125 >> loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/special_tokens_map.json from cache at /root/.cache/huggingface/transformers/6c3239a63aaf46ec7625b38abfe41fc2ce0b25f90800aefe6526256340d4ab6d.2b8bf81243d08385c806171bc7ced6d2a0dcc7f896ca637f4e777418f7f0cc3c\n",
            "[INFO|tokenization_utils_base.py:1717] 2021-05-25 03:17:08,125 >> loading file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/tokenizer_config.json from cache at /root/.cache/huggingface/transformers/3cc88b3aa29bb2546db2dc21783292e2a086bb7158c7b5ceddeb24158a85c183.e74f7c3643ee79eb023ead36008be72fe726dada60fa3b2a0569925cfefa1e74\n",
            "[INFO|file_utils.py:1556] 2021-05-25 03:17:08,452 >> https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/pytorch_model.bin not found in cache or force_download set to True, downloading to /root/.cache/huggingface/transformers/tmpsq4pv9ee\n",
            "Downloading: 100% 526M/526M [00:16<00:00, 31.7MB/s]\n",
            "[INFO|file_utils.py:1560] 2021-05-25 03:17:25,096 >> storing https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/pytorch_model.bin in cache at /root/.cache/huggingface/transformers/b0ace3b93ace62067a246888f1e54e2d3ec20807d4d3e27ac602eef3b7091c0b.6525df88f1d5a2d33d95ce2458ef6af9658fe7d1393d6707e0e318779ccc68ff\n",
            "[INFO|file_utils.py:1568] 2021-05-25 03:17:25,096 >> creating metadata file for /root/.cache/huggingface/transformers/b0ace3b93ace62067a246888f1e54e2d3ec20807d4d3e27ac602eef3b7091c0b.6525df88f1d5a2d33d95ce2458ef6af9658fe7d1393d6707e0e318779ccc68ff\n",
            "[INFO|modeling_utils.py:1155] 2021-05-25 03:17:25,097 >> loading weights file https://huggingface.co/EleutherAI/gpt-neo-125M/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/b0ace3b93ace62067a246888f1e54e2d3ec20807d4d3e27ac602eef3b7091c0b.6525df88f1d5a2d33d95ce2458ef6af9658fe7d1393d6707e0e318779ccc68ff\n",
            "[INFO|modeling_utils.py:1339] 2021-05-25 03:17:27,160 >> All model checkpoint weights were used when initializing GPTNeoForCausalLM.\n",
            "\n",
            "[INFO|modeling_utils.py:1348] 2021-05-25 03:17:27,160 >> All the weights of GPTNeoForCausalLM were initialized from the model checkpoint at EleutherAI/gpt-neo-125M.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPTNeoForCausalLM for predictions without further training.\n",
            "  0% 0/1 [00:00<?, ?ba/s][WARNING|tokenization_utils_base.py:3171] 2021-05-25 03:17:49,097 >> Token indices sequence length is longer than the specified maximum sequence length for this model (11625841 > 2048). Running this sequence through the model will result in indexing errors\n",
            "[WARNING|run_clm.py:331] 2021-05-25 03:17:49,097 >> ^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits before being passed to the model.\n",
            "100% 1/1 [00:23<00:00, 23.45s/ba]\n",
            "100% 1/1 [00:04<00:00,  4.53s/ba]\n",
            "05/25/2021 03:17:55 - WARNING - __main__ -   The tokenizer picked seems to have a very large `model_max_length` (2048). Picking 1024 instead. You can change that default value by passing --block_size xxx.\n",
            "100% 1/1 [00:09<00:00,  9.63s/ba]\n",
            "100% 1/1 [00:02<00:00,  2.08s/ba]\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "[INFO|trainer.py:415] 2021-05-25 03:18:18,372 >> Using amp fp16 backend\n",
            "[INFO|trainer.py:1143] 2021-05-25 03:18:18,383 >> ***** Running training *****\n",
            "[INFO|trainer.py:1144] 2021-05-25 03:18:18,383 >>   Num examples = 11353\n",
            "[INFO|trainer.py:1145] 2021-05-25 03:18:18,383 >>   Num Epochs = 5\n",
            "[INFO|trainer.py:1146] 2021-05-25 03:18:18,383 >>   Instantaneous batch size per device = 1\n",
            "[INFO|trainer.py:1147] 2021-05-25 03:18:18,383 >>   Total train batch size (w. parallel, distributed & accumulation) = 1\n",
            "[INFO|trainer.py:1148] 2021-05-25 03:18:18,383 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1149] 2021-05-25 03:18:18,383 >>   Total optimization steps = 56765\n",
            "[INFO|integrations.py:675] 2021-05-25 03:18:18,384 >> Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mgagan3012\u001b[0m (use `wandb login --relogin` to force relogin)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "2021-05-25 03:18:19.580867: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.11.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.10.30\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33m/content/model\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/gagan3012/project-code-py\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/gagan3012/project-code-py/runs/2opvo3dp\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in /content/transformers/examples/pytorch/language-modeling/wandb/run-20210525_031818-2opvo3dp\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run `wandb offline` to turn off syncing.\n",
            "\n",
            "  1%|▎                                    | 500/56765 [02:01<4:48:26,  3.25it/s]{'loss': 2.0275, 'learning_rate': 4.9562230247511674e-05, 'epoch': 0.04}\n",
            "{'loss': 2.0037, 'learning_rate': 4.912269884611997e-05, 'epoch': 0.09}\n",
            "{'loss': 1.9314, 'learning_rate': 4.868228662027658e-05, 'epoch': 0.13}\n",
            "  4%|█▎                                  | 2000/56765 [08:23<4:35:14,  3.32it/s]{'loss': 1.9222, 'learning_rate': 4.824187439443319e-05, 'epoch': 0.18}\n",
            "  4%|█▌                                  | 2500/56765 [10:30<4:36:18,  3.27it/s]{'loss': 1.9162, 'learning_rate': 4.78014621685898e-05, 'epoch': 0.22}\n",
            "                                                                                {'loss': 1.77, 'learning_rate': 4.736104994274641e-05, 'epoch': 0.26}\n",
            "  6%|██▏                                 | 3500/56765 [14:44<4:27:20,  3.32it/s]{'loss': 1.7844, 'learning_rate': 4.692151854135471e-05, 'epoch': 0.31}\n",
            "  7%|██▌                                 | 4000/56765 [16:52<4:25:17,  3.31it/s]{'loss': 1.8885, 'learning_rate': 4.648110631551132e-05, 'epoch': 0.35}\n",
            "  8%|██▊                                 | 4500/56765 [18:59<4:24:03,  3.30it/s]{'loss': 1.7094, 'learning_rate': 4.604069408966793e-05, 'epoch': 0.4}\n",
            "  9%|███▏                                | 5000/56765 [21:06<4:19:46,  3.32it/s]{'loss': 1.8179, 'learning_rate': 4.560028186382454e-05, 'epoch': 0.44}\n",
            "{'loss': 1.8352, 'learning_rate': 4.515986963798115e-05, 'epoch': 0.48}\n",
            "{'loss': 1.7946, 'learning_rate': 4.471945741213776e-05, 'epoch': 0.53}\n",
            "{'loss': 1.7258, 'learning_rate': 4.427904518629437e-05, 'epoch': 0.57}\n",
            "{'loss': 1.671, 'learning_rate': 4.383863296045098e-05, 'epoch': 0.62}\n",
            "{'loss': 1.8649, 'learning_rate': 4.339910155905928e-05, 'epoch': 0.66}\n",
            "{'loss': 1.9734, 'learning_rate': 4.295868933321589e-05, 'epoch': 0.7}\n",
            " 15%|█████▍                              | 8500/56765 [35:56<4:02:19,  3.32it/s]{'loss': 1.8717, 'learning_rate': 4.25182771073725e-05, 'epoch': 0.75}\n",
            "{'loss': 1.7005, 'learning_rate': 4.207786488152911e-05, 'epoch': 0.79}\n",
            "{'loss': 1.6136, 'learning_rate': 4.163745265568572e-05, 'epoch': 0.84}\n",
            " 18%|██████▏                            | 10000/56765 [42:18<3:56:24,  3.30it/s]{'loss': 1.7965, 'learning_rate': 4.1197040429842334e-05, 'epoch': 0.88}\n",
            "{'loss': 1.6852, 'learning_rate': 4.075662820399894e-05, 'epoch': 0.92}\n",
            "{'loss': 1.5454, 'learning_rate': 4.031709680260724e-05, 'epoch': 0.97}\n",
            "{'loss': 1.6019, 'learning_rate': 3.987668457676385e-05, 'epoch': 1.01}\n",
            "{'loss': 1.7469, 'learning_rate': 3.943627235092046e-05, 'epoch': 1.06}\n",
            "{'loss': 1.8031, 'learning_rate': 3.8995860125077074e-05, 'epoch': 1.1}\n",
            "{'loss': 1.6901, 'learning_rate': 3.855544789923368e-05, 'epoch': 1.15}\n",
            " 24%|████████▎                          | 13500/56765 [57:06<3:37:58,  3.31it/s]{'loss': 1.6653, 'learning_rate': 3.811503567339029e-05, 'epoch': 1.19}\n",
            "{'loss': 1.6627, 'learning_rate': 3.767462344754691e-05, 'epoch': 1.23}\n",
            " 26%|████████▍                        | 14500/56765 [1:01:20<3:31:14,  3.33it/s]{'loss': 1.6196, 'learning_rate': 3.723421122170352e-05, 'epoch': 1.28}\n",
            "{'loss': 1.7333, 'learning_rate': 3.679379899586013e-05, 'epoch': 1.32}\n",
            "{'loss': 1.6364, 'learning_rate': 3.635426759446842e-05, 'epoch': 1.37}\n",
            "{'loss': 1.6258, 'learning_rate': 3.591385536862503e-05, 'epoch': 1.41}\n",
            "{'loss': 1.7038, 'learning_rate': 3.547344314278164e-05, 'epoch': 1.45}\n",
            " 30%|█████████▉                       | 17000/56765 [1:11:55<3:20:14,  3.31it/s]{'loss': 1.6104, 'learning_rate': 3.5033030916938255e-05, 'epoch': 1.5}\n",
            " 31%|██████████▏                      | 17500/56765 [1:14:02<3:17:34,  3.31it/s]{'loss': 1.707, 'learning_rate': 3.459438033999824e-05, 'epoch': 1.54}\n",
            " 32%|██████████▍                      | 18000/56765 [1:16:09<3:14:32,  3.32it/s]{'loss': 1.6641, 'learning_rate': 3.415396811415485e-05, 'epoch': 1.59}\n",
            " 33%|██████████▊                      | 18500/56765 [1:18:16<3:11:48,  3.32it/s]{'loss': 1.7144, 'learning_rate': 3.371355588831146e-05, 'epoch': 1.63}\n",
            " 33%|███████████                      | 19000/56765 [1:20:24<3:10:37,  3.30it/s]{'loss': 1.7934, 'learning_rate': 3.327314366246807e-05, 'epoch': 1.67}\n",
            "{'loss': 1.6137, 'learning_rate': 3.283273143662468e-05, 'epoch': 1.72}\n",
            "{'loss': 1.6195, 'learning_rate': 3.239231921078129e-05, 'epoch': 1.76}\n",
            "{'loss': 1.6228, 'learning_rate': 3.1951906984937905e-05, 'epoch': 1.81}\n",
            "{'loss': 1.5511, 'learning_rate': 3.151149475909451e-05, 'epoch': 1.85}\n",
            "{'loss': 1.5842, 'learning_rate': 3.107196335770281e-05, 'epoch': 1.89}\n",
            "{'loss': 1.6081, 'learning_rate': 3.063155113185942e-05, 'epoch': 1.94}\n",
            " 40%|█████████████                    | 22500/56765 [1:35:15<2:52:21,  3.31it/s]{'loss': 1.6629, 'learning_rate': 3.0191138906016033e-05, 'epoch': 1.98}\n",
            " 41%|█████████████▎                   | 23000/56765 [1:37:22<2:50:26,  3.30it/s]{'loss': 1.6513, 'learning_rate': 2.975072668017264e-05, 'epoch': 2.03}\n",
            "{'loss': 1.5899, 'learning_rate': 2.9310314454329253e-05, 'epoch': 2.07}\n",
            "{'loss': 1.6249, 'learning_rate': 2.8869902228485862e-05, 'epoch': 2.11}\n",
            "{'loss': 1.6052, 'learning_rate': 2.8429490002642474e-05, 'epoch': 2.16}\n",
            "                                                                                {'loss': 1.5361, 'learning_rate': 2.7989077776799083e-05, 'epoch': 2.2}\n",
            " 45%|██████████████▊                  | 25500/56765 [1:47:58<2:38:04,  3.30it/s]{'loss': 1.6174, 'learning_rate': 2.7548665550955695e-05, 'epoch': 2.25}\n",
            " 46%|███████████████                  | 26000/56765 [1:50:05<2:35:45,  3.29it/s]{'loss': 1.5097, 'learning_rate': 2.7108253325112303e-05, 'epoch': 2.29}\n",
            " 47%|███████████████▍                 | 26500/56765 [1:52:13<2:32:04,  3.32it/s]{'loss': 1.4344, 'learning_rate': 2.66687219237206e-05, 'epoch': 2.33}\n",
            "{'loss': 1.6261, 'learning_rate': 2.6228309697877214e-05, 'epoch': 2.38}\n",
            " 48%|███████████████▉                 | 27500/56765 [1:56:27<2:27:10,  3.31it/s]{'loss': 1.5999, 'learning_rate': 2.5787897472033822e-05, 'epoch': 2.42}\n",
            " 49%|████████████████▎                | 28000/56765 [1:58:34<2:25:37,  3.29it/s]{'loss': 1.5727, 'learning_rate': 2.5347485246190434e-05, 'epoch': 2.47}\n",
            "{'loss': 1.5557, 'learning_rate': 2.4907953844798733e-05, 'epoch': 2.51}\n",
            "{'loss': 1.7235, 'learning_rate': 2.4467541618955345e-05, 'epoch': 2.55}\n",
            "{'loss': 1.5407, 'learning_rate': 2.4027129393111957e-05, 'epoch': 2.6}\n",
            " 53%|█████████████████▍               | 30000/56765 [2:07:04<2:15:33,  3.29it/s]{'loss': 1.5626, 'learning_rate': 2.3586717167268565e-05, 'epoch': 2.64}\n",
            "{'loss': 1.6137, 'learning_rate': 2.3146304941425177e-05, 'epoch': 2.69}\n",
            " 55%|██████████████████               | 31000/56765 [2:11:18<2:10:23,  3.29it/s]{'loss': 1.6471, 'learning_rate': 2.2706773540033472e-05, 'epoch': 2.73}\n",
            " 55%|██████████████████▎              | 31500/56765 [2:13:26<2:07:17,  3.31it/s]{'loss': 1.685, 'learning_rate': 2.226636131419008e-05, 'epoch': 2.77}\n",
            " 56%|██████████████████▌              | 32000/56765 [2:15:33<2:05:31,  3.29it/s]{'loss': 1.5579, 'learning_rate': 2.1825949088346693e-05, 'epoch': 2.82}\n",
            "{'loss': 1.5701, 'learning_rate': 2.1385536862503305e-05, 'epoch': 2.86}\n",
            "{'loss': 1.6108, 'learning_rate': 2.0945124636659917e-05, 'epoch': 2.91}\n",
            " 59%|███████████████████▍             | 33500/56765 [2:21:55<1:57:05,  3.31it/s]{'loss': 1.6338, 'learning_rate': 2.0505593235268212e-05, 'epoch': 2.95}\n",
            "{'loss': 1.623, 'learning_rate': 2.006518100942482e-05, 'epoch': 2.99}\n",
            " 61%|████████████████████             | 34500/56765 [2:26:10<1:52:06,  3.31it/s]{'loss': 1.5039, 'learning_rate': 1.9624768783581433e-05, 'epoch': 3.04}\n",
            "{'loss': 1.6072, 'learning_rate': 1.918435655773804e-05, 'epoch': 3.08}\n",
            "{'loss': 1.5008, 'learning_rate': 1.8743944331894657e-05, 'epoch': 3.13}\n",
            " 63%|████████████████████▉            | 36000/56765 [2:32:32<1:45:09,  3.29it/s]{'loss': 1.5741, 'learning_rate': 1.8303532106051265e-05, 'epoch': 3.17}\n",
            " 64%|█████████████████████▏           | 36500/56765 [2:34:39<1:42:02,  3.31it/s]{'loss': 1.4937, 'learning_rate': 1.7863119880207877e-05, 'epoch': 3.22}\n",
            "{'loss': 1.5352, 'learning_rate': 1.7423588478816172e-05, 'epoch': 3.26}\n",
            "{'loss': 1.6432, 'learning_rate': 1.698317625297278e-05, 'epoch': 3.3}\n",
            " 67%|██████████████████████           | 38000/56765 [2:41:01<1:34:06,  3.32it/s]{'loss': 1.5811, 'learning_rate': 1.6542764027129393e-05, 'epoch': 3.35}\n",
            "{'loss': 1.5633, 'learning_rate': 1.6102351801286005e-05, 'epoch': 3.39}\n",
            "{'loss': 1.5949, 'learning_rate': 1.5661939575442617e-05, 'epoch': 3.44}\n",
            "{'loss': 1.5309, 'learning_rate': 1.5222408174050912e-05, 'epoch': 3.48}\n",
            " 70%|███████████████████████▎         | 40000/56765 [2:49:30<1:24:00,  3.33it/s]{'loss': 1.4779, 'learning_rate': 1.4781995948207522e-05, 'epoch': 3.52}\n",
            " 71%|███████████████████████▌         | 40500/56765 [2:51:37<1:21:40,  3.32it/s]{'loss': 1.5713, 'learning_rate': 1.4341583722364133e-05, 'epoch': 3.57}\n",
            "{'loss': 1.521, 'learning_rate': 1.3901171496520743e-05, 'epoch': 3.61}\n",
            " 73%|████████████████████████▏        | 41500/56765 [2:55:52<1:16:27,  3.33it/s]{'loss': 1.558, 'learning_rate': 1.3461640095129042e-05, 'epoch': 3.66}\n",
            "{'loss': 1.5095, 'learning_rate': 1.3021227869285652e-05, 'epoch': 3.7}\n",
            " 75%|████████████████████████▋        | 42500/56765 [3:00:07<1:11:45,  3.31it/s]{'loss': 1.4322, 'learning_rate': 1.2580815643442262e-05, 'epoch': 3.74}\n",
            " 76%|████████████████████████▉        | 43000/56765 [3:02:14<1:09:26,  3.30it/s]{'loss': 1.558, 'learning_rate': 1.2140403417598874e-05, 'epoch': 3.79}\n",
            "{'loss': 1.5619, 'learning_rate': 1.1700872016207171e-05, 'epoch': 3.83}\n",
            "{'loss': 1.4401, 'learning_rate': 1.1260459790363781e-05, 'epoch': 3.88}\n",
            "{'loss': 1.5182, 'learning_rate': 1.0820047564520392e-05, 'epoch': 3.92}\n",
            "{'loss': 1.5103, 'learning_rate': 1.0379635338677002e-05, 'epoch': 3.96}\n",
            " 80%|████████████████████████████       | 45500/56765 [3:12:51<56:23,  3.33it/s]{'loss': 1.5342, 'learning_rate': 9.939223112833612e-06, 'epoch': 4.01}\n",
            " 81%|████████████████████████████▎      | 46000/56765 [3:14:58<54:17,  3.30it/s]{'loss': 1.4571, 'learning_rate': 9.498810886990224e-06, 'epoch': 4.05}\n",
            " 82%|████████████████████████████▋      | 46500/56765 [3:17:05<51:27,  3.32it/s]{'loss': 1.4884, 'learning_rate': 9.059279485598521e-06, 'epoch': 4.1}\n",
            " 83%|████████████████████████████▉      | 47000/56765 [3:19:12<49:21,  3.30it/s]{'loss': 1.5288, 'learning_rate': 8.618867259755131e-06, 'epoch': 4.14}\n",
            "{'loss': 1.6157, 'learning_rate': 8.178455033911742e-06, 'epoch': 4.18}\n",
            "{'loss': 1.5245, 'learning_rate': 7.738042808068352e-06, 'epoch': 4.23}\n",
            "{'loss': 1.4058, 'learning_rate': 7.29851140667665e-06, 'epoch': 4.27}\n",
            "{'loss': 1.617, 'learning_rate': 6.858099180833261e-06, 'epoch': 4.32}\n",
            "{'loss': 1.4707, 'learning_rate': 6.417686954989871e-06, 'epoch': 4.36}\n",
            "{'loss': 1.4786, 'learning_rate': 5.977274729146481e-06, 'epoch': 4.4}\n",
            "{'loss': 1.4711, 'learning_rate': 5.536862503303092e-06, 'epoch': 4.45}\n",
            "{'loss': 1.4751, 'learning_rate': 5.097331101911389e-06, 'epoch': 4.49}\n",
            "{'loss': 1.5495, 'learning_rate': 4.6569188760679995e-06, 'epoch': 4.54}\n",
            " 92%|████████████████████████████████   | 52000/56765 [3:40:26<24:10,  3.28it/s]{'loss': 1.4748, 'learning_rate': 4.21650665022461e-06, 'epoch': 4.58}\n",
            "{'loss': 1.4816, 'learning_rate': 3.7760944243812214e-06, 'epoch': 4.62}\n",
            " 93%|████████████████████████████████▋  | 53000/56765 [3:44:40<18:58,  3.31it/s]{'loss': 1.5052, 'learning_rate': 3.3356821985378317e-06, 'epoch': 4.67}\n",
            "{'loss': 1.4555, 'learning_rate': 2.896150797146129e-06, 'epoch': 4.71}\n",
            "{'loss': 1.4899, 'learning_rate': 2.4557385713027396e-06, 'epoch': 4.76}\n",
            "{'loss': 1.4849, 'learning_rate': 2.01532634545935e-06, 'epoch': 4.8}\n",
            " 97%|█████████████████████████████████▉ | 55000/56765 [3:53:10<08:55,  3.30it/s]{'loss': 1.5955, 'learning_rate': 1.5749141196159605e-06, 'epoch': 4.84}\n",
            " 98%|██████████████████████████████████▏| 55500/56765 [3:55:17<06:23,  3.30it/s]{'loss': 1.4426, 'learning_rate': 1.1345018937725712e-06, 'epoch': 4.89}\n",
            " 99%|██████████████████████████████████▌| 56000/56765 [3:57:25<03:50,  3.32it/s]{'loss': 1.4316, 'learning_rate': 6.940896679291817e-07, 'epoch': 4.93}\n",
            "{'loss': 1.5447, 'learning_rate': 2.5367744208579233e-07, 'epoch': 4.98}\n",
            "100%|███████████████████████████████████| 56765/56765 [4:00:39<00:00,  3.94it/s][INFO|trainer.py:1339] 2021-05-25 07:19:00,838 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "100%|███████████████████████████████████| 56765/56765 [4:00:39<00:00,  3.94it/s]{'train_runtime': 14442.4553, 'train_samples_per_second': 3.93, 'train_steps_per_second': 3.93, 'epoch': 5.0}\n",
            "100%|███████████████████████████████████| 56765/56765 [4:00:39<00:00,  3.93it/s]\n",
            "[INFO|trainer.py:1884] 2021-05-25 07:19:00,844 >> Saving model checkpoint to /content/model\n",
            "[INFO|configuration_utils.py:351] 2021-05-25 07:19:00,846 >> Configuration saved in /content/model/config.json\n",
            "[INFO|modeling_utils.py:889] 2021-05-25 07:19:02,317 >> Model weights saved in /content/model/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:1924] 2021-05-25 07:19:02,319 >> tokenizer config file saved in /content/model/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:1930] 2021-05-25 07:19:02,320 >> Special tokens file saved in /content/model/special_tokens_map.json\n",
            "[INFO|trainer_pt_utils.py:907] 2021-05-25 07:19:02,415 >> ***** train metrics *****\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:19:02,415 >>   epoch                    =        5.0\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:19:02,415 >>   train_runtime            = 4:00:42.45\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:19:02,415 >>   train_samples            =      11353\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:19:02,416 >>   train_samples_per_second =       3.93\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:19:02,416 >>   train_steps_per_second   =       3.93\n",
            "05/25/2021 07:19:02 - INFO - __main__ -   *** Evaluate ***\n",
            "[INFO|trainer.py:2130] 2021-05-25 07:19:02,425 >> ***** Running Evaluation *****\n",
            "[INFO|trainer.py:2132] 2021-05-25 07:19:02,425 >>   Num examples = 2598\n",
            "[INFO|trainer.py:2135] 2021-05-25 07:19:02,425 >>   Batch size = 1\n",
            "100%|███████████████████████████████████████| 2598/2598 [05:30<00:00,  7.86it/s]\n",
            "[INFO|trainer_pt_utils.py:907] 2021-05-25 07:24:33,072 >> ***** eval metrics *****\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   epoch                   =        5.0\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   eval_loss               =     0.8997\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   eval_runtime            = 0:05:30.64\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   eval_samples            =       2598\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   eval_samples_per_second =      7.857\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   eval_steps_per_second   =      7.857\n",
            "[INFO|trainer_pt_utils.py:912] 2021-05-25 07:24:33,072 >>   perplexity              =     2.4589\n",
            "\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Waiting for W&B process to finish, PID 1108\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Program ended successfully.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                                                \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find user logs for this run at: /content/transformers/examples/pytorch/language-modeling/wandb/run-20210525_031818-2opvo3dp/logs/debug.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find internal logs for this run at: /content/transformers/examples/pytorch/language-modeling/wandb/run-20210525_031818-2opvo3dp/logs/debug-internal.log\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                  train/loss 1.5447\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                         train/learning_rate 0.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                 train/epoch 5.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                           train/global_step 56765\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                    _runtime 14775\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                  _timestamp 1621927473\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                       _step 114\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                         train/train_runtime 14442.4553\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                              train/train_samples_per_second 3.93\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                train/train_steps_per_second 3.93\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                            train/total_flos 4.366478177206272e+16\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                   eval/loss 0.8997\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                                eval/runtime 330.6456\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                     eval/samples_per_second 7.857\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                                       eval/steps_per_second 7.857\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                       train/loss █▇▅▄▅▆▄▄▃▄▃▃▄▅▃▃▃▃▂▃▄▃▂▃▂▂▃▃▂▂▂▂▁▃▃▁▁▁▃▂\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              train/learning_rate ████▇▇▇▇▇▆▆▆▆▆▅▅▅▅▅▅▄▄▄▄▄▄▃▃▃▃▃▂▂▂▂▂▂▁▁▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                      train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇████\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                train/global_step ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇████\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                         _runtime ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                       _timestamp ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                            _step ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:              train/train_runtime ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   train/train_samples_per_second ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train/train_steps_per_second ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                 train/total_flos ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                        eval/loss ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:                     eval/runtime ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          eval/samples_per_second ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:            eval/steps_per_second ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 1 media file(s), 0 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced \u001b[33m/content/model\u001b[0m: \u001b[34mhttps://wandb.ai/gagan3012/project-code-py/runs/2opvo3dp\u001b[0m\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vf7R7mTZggCo"
      },
      "source": [
        ""
      ],
      "execution_count": 25,
      "outputs": []
    }
  ]
}